Flow analysis exploration

Event logs studied:

logs = read.csv("../logs_summary.csv")
DT::datatable(logs, filter = "top")

Training configuration:

• 2/3 x 1/3 temporal split
• zero bucketing
• aggregate encoding
• XGboost with hyperparam optimization (mainly, # of trees)

Surveyed techniques:

• predictive flow analysis (FA) - single and multiple model options (zero bucketing and prefix-length bucketing, respectively)
• adaptive FA (based on a variable threshold e.g. min N^2 samples in the training set, where N is the number of features )
• mean FA - each cycle time and branching probability is derived based on historical averages

Baselines

• simple regression - remaining time is predicted as a single value (comparable to Massimiliano de Leoni’s approach)
• Wil
• Andreas Rogge-Solti.

filenames <- list.files()[grep(paste("^validation_Aalst_(?=.*\\.csv)", sep=''), list.files(), perl=TRUE)]
for (filename in filenames) {
  this = read.csv(filename, sep=",", header = TRUE)
  this = this[this$nr_events < 11,]
  #this = this[this$metric == "mae",]
  this$nr_cases = (this$nr_cases/sum(this$nr_cases))
  this$score_weighted = this$score * this$nr_cases
  this$method2 = this$cls
  this$method3 = ""
  this$filename = filename
  dat = rbind(dat, this)
}

filenames <- list.files()[grep(paste("^validation_SPN_(?=.*\\.csv)", sep=''), list.files(), perl=TRUE)]
for (filename in filenames) {
  this = read.csv(filename, sep=",", header = TRUE)
  this = this[this$nr_events < 11,]
  #this = this[this$metric == "mae",]
  this$nr_cases = this$nr_cases/sum(this$nr_cases)
  this$score_weighted = this$score * this$nr_cases
  this$method2 = this$cls
  this$method3 = ""
  this$filename = filename
  dat = rbind(dat, this)
}

Average MAE across all prefix lengths for all datasets and methods:

agg_MAE_unw = as.data.frame(aggregate(dat$score,by=list(dataset=dat$dataset, method=dat$method), mean))
agg_MAE_w = as.data.frame(aggregate(dat$score_weighted,by=list(dataset=dat$dataset, method=dat$method), sum))
agg_MAE_unw[,3] = round(agg_MAE_unw[,3], 3)
agg_MAE_w[,3] = round(agg_MAE_w[,3], 3)
colnames(agg_MAE_unw)[3] = "Avg_unweighted_MAE_days"
colnames(agg_MAE_w)[3] = "Avg_weighted_MAE_days"
agg_MAE = merge(agg_MAE_w, agg_MAE_unw)
agg_MAE = agg_MAE[with(agg_MAE, order(dataset, Avg_weighted_MAE_days, method = "radix")), ]
DT::datatable(agg_MAE, filter = "top")

tmp = ddply(dat, .(dataset,method), summarize, std=sd(score))
tmp$std = round(tmp$std,3)
agg_MAE = merge(agg_MAE, tmp)
agg_MAE$Avg_unweighted_MAE_days=NULL
agg_MAE = agg_MAE[with(agg_MAE, order(dataset, Avg_weighted_MAE_days, method = "radix")), ]
DT::datatable(agg_MAE, filter = "top")

Plots MAE vs prefix size

dat$method[dat$method=="_best_Aalst"] = "TS (best)"
dat$method[dat$method=="adaptive_FA_zero_combined"] = "adaptive FA"
dat$method[dat$method=="mean_FA_zero_combined"] = "mean FA"
dat$method[dat$method=="predictive_FA_prefix_index"] = "predictive FA (multiple)"
dat$method[dat$method=="predictive_FA_zero_combined"] = "predictive FA (single)"
dat$method[dat$method=="simple_regression_prefix_index"] = "black box (multiple)"
dat$method[dat$method=="simple_regression_zero_combined"] = "black box (single)"
dat$method[dat$method=="_SPN_Rogge-Solti"] = "stochastic Petri net"
levels(dat$dataset)[1] = "BPIC'12 A"
levels(dat$dataset)[2] = "BPIC'12 O"
levels(dat$dataset)[3] = "BPIC'12 W_n|1|"
levels(dat$dataset)[4] = "BPIC'12 W"
levels(dat$dataset)[5] = "CR"
levels(dat$dataset)[6] = "Helpdesk"
levels(dat$dataset)[7] = "Hospital"
levels(dat$dataset)[8] = "Invoice"
levels(dat$dataset)[9] = "RTFMP"
for (df in levels(dat$dataset)) {
  p = ggplot(dat[dat$dataset == df,], aes(x = nr_events, y = score, color = method)) + geom_point() + geom_line() + 
    #facet_wrap(~ dataset, ncol = 3, scales = "free") + 
    ylab("MAE, days") + xlab("Prefix length")+
    ggtitle(df)
    theme(legend.position="top")+
    theme(panel.background = element_rect(fill = 'white', colour = 'black',size=0.5)) + 
    theme(panel.grid.major = element_line(colour = 'lightgrey', size = 0.3))+
  theme(legend.background = element_rect(colour = 'white',size = 0.1, linetype='solid'))+
  theme(legend.title=element_blank())
  #print(p)
}

Interactive plots can be accessed here

for paper only

agg_MAE$method[agg_MAE$method=="_best_Aalst"] = "TS (best)"
agg_MAE$method[agg_MAE$method=="adaptive_FA_zero_combined"] = "adaptive FA"
agg_MAE$method[agg_MAE$method=="mean_FA_zero_combined"] = "mean FA"
agg_MAE$method[agg_MAE$method=="predictive_FA_prefix_index"] = "predictive FA (multiple)"
agg_MAE$method[agg_MAE$method=="predictive_FA_zero_combined"] = "predictive FA (single)"
agg_MAE$method[agg_MAE$method=="simple_regression_prefix_index"] = "black box (multiple)"
agg_MAE$method[agg_MAE$method=="simple_regression_zero_combined"] = "black box (single)"
agg_MAE$method[agg_MAE$method=="_SPN_Rogge-Solti"] = "stochastic Petri net"
dat = data.frame()
for (dset in levels(agg_MAE$dataset)) {
  df = agg_MAE[agg_MAE$dataset==dset,]
  df$Order = rank(df$Avg_weighted_MAE_days,ties.method = "min")
  dat = rbind(dat, df)
}
dat_ = ddply(dat, .(method), summarize, Mean=mean(Order))
foo = ddply(dat, .(method), summarize, std=sd(Order))
dat_ = merge(dat_,foo)
dat_ = dat_[order(dat_$Mean),]

pdf(file="mean-ranking.pdf",family="CM Roman",width=4,height=2)
p=ggplot(dat_, aes(x=reorder(method, -Mean), y=Mean, fill=method, group=method)) + 
  geom_bar(stat="identity") + theme(text=element_text(size=24))+
  geom_errorbar(aes(ymin=Mean-std, ymax=Mean+std),width=.2) + 
  theme_bw() + coord_flip() + theme(legend.position="none") + ylab("mean ranking")+
  theme(axis.title.y=element_blank(),axis.text=element_text(color="black"),axis.ticks.y=element_blank())+
  scale_y_continuous(expand = c(0,0), limits=c(0,9),breaks = seq(1,7,2), labels =as.character(seq(1,7,2)))+
  scale_fill_manual(values=cbPalette)
print(p)
dev.off()

null device 
          1 

embed_fonts("mean-ranking.pdf",outfile="mean-ranking.pdf")
ggplotly(p)

dat$case_duration = 0
dat$case_duration[dat$dataset=="BPI2012A"] = 7.5
dat$case_duration[dat$dataset=="BPI2012O"] = 15.1
dat$case_duration[dat$dataset=="BPI2012W"] = 11.4
dat$case_duration[dat$dataset=="BPI2012W_no_dup"] = 11.4
dat$case_duration[dat$dataset=="CreditRequirement"] = 0.95
dat$case_duration[dat$dataset=="helpdesk"] = 7.3
dat$case_duration[dat$dataset=="minit_invoice_10"] = 2.225
dat$case_duration[dat$dataset=="traffic_fines_139"] = 582
dat$case_duration[dat$dataset=="hospital_billing_977"] = 165.2
dat$Avg_weighted_MAE_days = dat$Avg_weighted_MAE_days / dat$case_duration
dat$std = dat$std / dat$case_duration
dat$Order = NULL

dat_ = ddply(dat, .(method), summarize, Mean=mean(Avg_weighted_MAE_days))
foo = ddply(dat, .(method), summarize, std=sd(Avg_weighted_MAE_days))
dat_ = merge(dat_,foo)
pdf(file="mean-ranking2.pdf",family="CM Roman",width=4,height=2)
p=ggplot(dat_, aes(x=reorder(method, -Mean), y=Mean, fill=method, group=method)) + 
  geom_bar(stat="identity") + theme(text=element_text(size=24))+
  geom_errorbar(aes(ymin=Mean-std, ymax=Mean+std),width=.2) + 
  theme_bw() + coord_flip() + theme(legend.position="none") + ylab("Average normalized MAE")+
  theme(axis.title.y=element_blank(),axis.text=element_text(color="black"),axis.ticks.y=element_blank())+
  scale_y_continuous(expand = c(0,0), limits=c(0,0.9), breaks = seq(0,0.8,0.2), labels =as.character(seq(0,0.8,0.2)))+
  scale_fill_manual(values=cbPalette)
print(p)
dev.off()

null device 
          1 

embed_fonts("mean-ranking2.pdf",outfile="mean-ranking2.pdf")
ggplotly(p)

dat_ = ddply(dat, .(method), summarize, Mean=mean(std))
foo = ddply(dat, .(method), summarize, Std=sd(std))
dat_ = merge(dat_,foo)
pdf(file="mean-ranking3.pdf",family="CM Roman",width=4,height=2)
p=ggplot(dat_, aes(x=reorder(method, -Mean), y=Mean, fill=method, group=method)) + 
  geom_bar(stat="identity") + theme(text=element_text(size=24))+
  geom_errorbar(aes(ymin=Mean-Std, ymax=Mean+Std),width=.2) + 
  theme_bw() + coord_flip() + theme(legend.position="none") + ylab("Average normalized standard deviation")+
  theme(axis.title.y=element_blank(),axis.text=element_text(color="black"),axis.ticks.y=element_blank())+
  scale_y_continuous(expand = c(0,0), limits=c(-0.02,0.45), breaks = seq(0,0.4,0.1), labels =as.character(seq(0,0.4,0.1)))+
  scale_fill_manual(values=cbPalette)
print(p)
dev.off()

null device 
          1 

embed_fonts("mean-ranking3.pdf",outfile="mean-ranking3.pdf")
ggplotly(p)